package com.shujia.base;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.ArrayList;

/*
    使用java模拟mr程序上传文件，进行切分的过程
    1、假设每一条学生数据为1M,总共898M数据
    2、将原本产生map任务的逻辑提前到生成block的阶段来
 */
public class LoadData {
    public static void main(String[] args) throws Exception {
        //读取数据
        BufferedReader br = new BufferedReader(new FileReader("D:\\projects\\IdeaProjects\\bigdata32\\hadoop\\data\\studentTest.txt"));
        int index = 0;
        //创建输出流对象
        BufferedWriter bw = new BufferedWriter(new FileWriter("D:\\projects\\IdeaProjects\\bigdata32\\hadoop\\data\\blocks\\block" + index));
        //创建变量计算读取到条数
        int rowNumber = 0;
        //创建一个集合存储读取的数据
        ArrayList<String> list = new ArrayList<>();
        String line = null;
        while ((line = br.readLine()) != null) {
            list.add(line);
            rowNumber++;
            if (rowNumber == 140) {
                //执行写操作,生成一个block块
                for (int i = index * 128; i < 128 * (index + 1); i++) {
                    bw.write(list.get(i));
                    bw.newLine();
                    bw.flush();
                }

                index++;
                rowNumber = 12;
                bw = new BufferedWriter(new FileWriter("D:\\projects\\IdeaProjects\\bigdata32\\hadoop\\data\\blocks\\block" + index));
            }
        }

        // 将剩下的数据写到一个block块中
        for (int i = list.size() - rowNumber; i < list.size(); i++) {
            bw.write(list.get(i));
            bw.newLine();
            bw.flush();
        }


        br.close();
    }
}
